# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter

from components.pipelines.public.mysql_pipeline import MysqlPipeline


class TaxPolicyDeclarationPipeline(MysqlPipeline):

    __item_dicts__ = []

    def process_item(self, item, spider):
        self.__item_dicts__.append(item)
        return item

    def dup_item(self, datas):
        keys = {}
        for item in datas:
            key = item["source_url"]
            if key not in keys:
                keys[key] = item
            else:
                src_item = keys[key]
                for k, v in item.items():
                    if k in ["policy_theme"]:
                        new_policy_theme = src_item[k].split("|")
                        new_policy_theme.append(v)
                        src_item[k] = "|".join(list(set(new_policy_theme)))
        last_datas = keys.values()
        return list(last_datas)

    def close_spider(self, spider):
        for item in self.dup_item(self.__item_dicts__):
            self.put_item(item)
        self.flush()
